{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lLGpv1fLLIjF"
   },
   "source": [
    "# Adalflow RAG Playbook example\n",
    "\n",
    "There are different patterns to build a RAG:\n",
    "\n",
    "- RAG with separate data process pipeline and a RAG task pipeline. This fits into a scenario where there is lots of data in production database, and we preprocess the data to embeddings and then we build a RAG task pipeline that retrieves context in multiple stages.\n",
    "\n",
    "- RAG with dynamic data access and caching the embedding dynamically in a local storage.\n",
    "\n",
    "Here we will have have a look at an example with a local DB using FAISS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "sfKEfaYC3Go7"
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "\n",
    "!pip install -U adalflow[openai,groq,faiss-cpu]\n",
    "\n",
    "clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip uninstall httpx anyio -y\n",
    "!pip install \"anyio>=3.1.0,<4.0\"\n",
    "!pip install httpx==0.24.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-4c_AGBt3PlR",
    "outputId": "a36f157b-0b18-4f3d-d5a8-09aa94743922"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please enter your OpenAI API key: ··········\n",
      "Please enter your GROQ API key: ··········\n",
      "API keys have been set.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from getpass import getpass\n",
    "\n",
    "# Prompt user to enter their API keys securely\n",
    "openai_api_key = getpass(\"Please enter your OpenAI API key: \")\n",
    "groq_api_key = getpass(\"Please enter your GROQ API key: \")\n",
    "\n",
    "# Set environment variables\n",
    "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
    "os.environ[\"GROQ_API_KEY\"] = groq_api_key\n",
    "\n",
    "print(\"API keys have been set.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "V9LsGDnm3RbV"
   },
   "outputs": [],
   "source": [
    "from typing import Any, List, Optional\n",
    "import os\n",
    "from adalflow.core import Component, Generator, Embedder, Sequential\n",
    "from adalflow.core.types import Document, ModelClientType\n",
    "from adalflow.core.string_parser import JsonParser\n",
    "from adalflow.core.db import LocalDB\n",
    "from adalflow.utils import setup_env\n",
    "from adalflow.components.retriever.faiss_retriever import FAISSRetriever\n",
    "from adalflow.components.data_process import (\n",
    "    RetrieverOutputToContextStr,\n",
    "    ToEmbeddings,\n",
    "    TextSplitter,\n",
    ")\n",
    "from adalflow.utils.global_config import get_adalflow_default_root_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "kWGTZxrw3Tli"
   },
   "outputs": [],
   "source": [
    "configs = {\n",
    "    \"embedder\": {\n",
    "        \"batch_size\": 100,\n",
    "        \"model_kwargs\": {\n",
    "            \"model\": \"text-embedding-3-small\",\n",
    "            \"dimensions\": 256,\n",
    "            \"encoding_format\": \"float\",\n",
    "        },\n",
    "    },\n",
    "    \"retriever\": {\n",
    "        \"top_k\": 5,\n",
    "    },\n",
    "    \"generator\": {\n",
    "        \"model_client\": ModelClientType.OPENAI(),\n",
    "        \"model_kwargs\": {\n",
    "            \"model\": \"gpt-3.5-turbo\",\n",
    "            \"temperature\": 0.3,\n",
    "            \"stream\": False,\n",
    "        },\n",
    "    },\n",
    "    \"text_splitter\": {\n",
    "        \"split_by\": \"word\",\n",
    "        \"chunk_size\": 400,\n",
    "        \"chunk_overlap\": 200,\n",
    "    },\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "1QE0PCKs4BLz"
   },
   "outputs": [],
   "source": [
    "def prepare_data_pipeline():\n",
    "    splitter = TextSplitter(**configs[\"text_splitter\"])\n",
    "    embedder = Embedder(\n",
    "        model_client=ModelClientType.OPENAI(),\n",
    "        model_kwargs=configs[\"embedder\"][\"model_kwargs\"],\n",
    "    )\n",
    "    embedder_transformer = ToEmbeddings(\n",
    "        embedder=embedder, batch_size=configs[\"embedder\"][\"batch_size\"]\n",
    "    )\n",
    "    data_transformer = Sequential(splitter, embedder_transformer)\n",
    "    return data_transformer\n",
    "\n",
    "\n",
    "def prepare_database_with_index(\n",
    "    docs: List[Document],\n",
    "    index_file: str = \"index.faiss\",\n",
    "    index_path: Optional[str] = None,\n",
    "):\n",
    "    index_path = index_path or get_adalflow_default_root_path()\n",
    "    index_path = os.path.join(index_path, index_file)\n",
    "    if os.path.exists(index_path):\n",
    "        return None\n",
    "    db = LocalDB()\n",
    "    db.load(docs)\n",
    "    data_transformer = prepare_data_pipeline()\n",
    "    db.transform(transformer=data_transformer, key=\"data_transformer\")\n",
    "    db.save_state(index_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "6Mu1HXhy4DIG"
   },
   "outputs": [],
   "source": [
    "RAG_PROMPT_TEMPLATE = r\"\"\"<START_OF_SYSTEM_MESSAGE>\n",
    "{{task_desc}}\n",
    "<END_OF_SYSTEM_MESSAGE>\n",
    "<START_OF_USER>\n",
    "{{input_str}}\n",
    "{{context_str}}\n",
    "<END_OF_USER>\n",
    "\"\"\"\n",
    "\n",
    "rag_prompt_task_desc = r\"\"\"\n",
    "You are a helpful assistant.\n",
    "\n",
    "Your task is to answer the query that may or may not come with context information.\n",
    "When context is provided, you should stick to the context and less on your prior knowledge to answer the query.\n",
    "\n",
    "Output JSON format:\n",
    "{\n",
    "    \"answer\": \"The answer to the query\",\n",
    "}\"\"\"\n",
    "\n",
    "\n",
    "class RAG(Component):\n",
    "    def __init__(\n",
    "        self,\n",
    "        index_file: str = \"index.faiss\",\n",
    "        index_path: Optional[str] = None,\n",
    "        configs: dict = configs,\n",
    "    ):\n",
    "        super().__init__()\n",
    "\n",
    "        index_path = index_path or get_adalflow_default_root_path()\n",
    "        index_path = os.path.join(index_path, index_file)\n",
    "        self.index_path = index_path\n",
    "\n",
    "        if not os.path.exists(index_path):\n",
    "            self.db = LocalDB()\n",
    "            self.register_data_transformer()\n",
    "            self.transformed_docs = []\n",
    "        else:\n",
    "            self.db = LocalDB.load_state(index_path)\n",
    "            self.transformed_docs = self.db.get_transformed_data(\"data_transformer\")\n",
    "\n",
    "        embedder = Embedder(\n",
    "            model_client=ModelClientType.OPENAI(),\n",
    "            model_kwargs=configs[\"embedder\"][\"model_kwargs\"],\n",
    "        )\n",
    "\n",
    "        self.retriever = FAISSRetriever(\n",
    "            **configs[\"retriever\"],\n",
    "            embedder=embedder,\n",
    "            documents=self.transformed_docs,\n",
    "            document_map_func=lambda doc: doc.vector,\n",
    "        )\n",
    "        self.retriever_output_processors = RetrieverOutputToContextStr(deduplicate=True)\n",
    "\n",
    "        self.generator = Generator(\n",
    "            **configs[\"generator\"],\n",
    "            prompt_kwargs={\"task_desc_str\": rag_prompt_task_desc},\n",
    "            output_processors=JsonParser(),\n",
    "        )\n",
    "\n",
    "    def register_data_transformer(self):\n",
    "        if \"data_transformer\" not in self.db.get_transformer_keys():\n",
    "            data_transformer = prepare_data_pipeline()\n",
    "            self.db.register_transformer(data_transformer, key=\"data_transformer\")\n",
    "            print(\"Data transformer registered\")\n",
    "\n",
    "    def add_documents(self, docs: List[Document]):\n",
    "        self.db.extend(docs, apply_transformer=True)\n",
    "        self.db.save_state(self.index_path)\n",
    "\n",
    "    def get_transformed_docs(self, filter_func=None):\n",
    "        return self.db.get_transformed_data(\"data_transformer\", filter_func)\n",
    "\n",
    "    def prepare_retriever(self, filter_func=None):\n",
    "        self.transformed_docs = self.get_transformed_docs(filter_func)\n",
    "        self.retriever.build_index_from_documents(\n",
    "            self.transformed_docs, document_map_func=lambda doc: doc.vector\n",
    "        )\n",
    "\n",
    "    def generate(self, query: str, context: Optional[str] = None) -> Any:\n",
    "        if not self.generator:\n",
    "            raise ValueError(\"Generator is not set\")\n",
    "        prompt_kwargs = {\"context_str\": context, \"input_str\": query}\n",
    "        response = self.generator(prompt_kwargs=prompt_kwargs)\n",
    "        return response, context\n",
    "\n",
    "    def call(self, query: str, verbose: bool = False) -> Any:\n",
    "        retrieved_documents = self.retriever(query)\n",
    "        for i, retriever_output in enumerate(retrieved_documents):\n",
    "            retrieved_documents[i].documents = [\n",
    "                self.transformed_docs[doc_index]\n",
    "                for doc_index in retriever_output.doc_indices\n",
    "            ]\n",
    "        if verbose:\n",
    "            print(f\"retrieved_documents: \\n {retrieved_documents}\")\n",
    "\n",
    "        context_str = self.retriever_output_processors(retrieved_documents)\n",
    "        if verbose:\n",
    "            print(f\"context_str: \\n {context_str}\")\n",
    "\n",
    "        return self.generate(query, context=context_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sPnx4PY34D1j",
    "outputId": "f66d6f1a-70bf-40e9-a160-591fcfdcbed3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Splitting Documents in Batches: 100%|██████████| 1/1 [00:00<00:00, 109.58it/s]\n",
      "Batch embedding documents: 100%|██████████| 1/1 [00:01<00:00,  1.33s/it]\n",
      "Adding embeddings to documents from batch: 1it [00:00, 6462.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved the state of the DB to /root/.adalflow/index.faiss\n",
      "RAG(\n",
      "  (db): LocalDB(name='LocalDB', items=[Document(id=doc1, text='My name is Li Yin, I love rock climbinglots of nonsense textlots of nonsense textlots of nonsense te...', meta_data={'title': \"Li Yin's profile\"}, vector=[], parent_doc_id=None, order=None, score=None), Document(id=doc2, text='lots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense ...', meta_data={'title': 'Interviewing Li Yin'}, vector=[], parent_doc_id=None, order=None, score=None)], transformed_items={'data_transformer': [Document(id=59f7f6ad-eb4c-4fdb-8d04-6dba1ee439bc, text='My name is Li Yin, I love rock climbinglots of nonsense textlots of nonsense textlots of nonsense te...', meta_data={'title': \"Li Yin's profile\"}, vector='len: 256', parent_doc_id=doc1, order=0, score=None), Document(id=2486725e-47ff-4978-84fc-7937778b0e45, text='textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nons...', meta_data={'title': \"Li Yin's profile\"}, vector='len: 256', parent_doc_id=doc1, order=1, score=None), Document(id=96993047-4cff-436d-b8ac-e02da4ae7fec, text='nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlot...', meta_data={'title': \"Li Yin's profile\"}, vector='len: 256', parent_doc_id=doc1, order=2, score=None), Document(id=77742f90-0c0c-4143-802d-3557577d4935, text='of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense text...', meta_data={'title': \"Li Yin's profile\"}, vector='len: 256', parent_doc_id=doc1, order=3, score=None), Document(id=81ba770e-c5f2-4dc5-98fc-349ab9143ef9, text='textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nons...', meta_data={'title': \"Li Yin's profile\"}, vector='len: 256', parent_doc_id=doc1, order=4, score=None), Document(id=dff6f5e3-5929-4e3c-ba5f-79f5116c1fa3, text='nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlot...', meta_data={'title': \"Li Yin's profile\"}, vector='len: 256', parent_doc_id=doc1, order=5, score=None), Document(id=1e7888e2-0783-40b2-ab85-067e3ba71fad, text='of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense text...', meta_data={'title': \"Li Yin's profile\"}, vector='len: 256', parent_doc_id=doc1, order=6, score=None), Document(id=2deb945f-dfb9-46d3-a60b-dae77e2f5fd8, text='lots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense ...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=0, score=None), Document(id=3d9c21aa-d583-47fe-b143-710b4bc4a8b2, text='textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonse...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=1, score=None), Document(id=a318ffea-2542-4493-ab2d-03d10a94e860, text='textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonse...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=2, score=None), Document(id=b5c05820-7545-43a8-a4a3-691c5ccc79d1, text='textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonse...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=3, score=None), Document(id=a739cd3e-8826-4e74-afa9-499498115621, text='textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonse...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=4, score=None), Document(id=7153cde2-b6ee-4485-91e9-9de2f4bd45ab, text='textLi Yin is an AI researcher and a software engineerlots of more nonsense textlots of more nonsens...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=5, score=None), Document(id=c3f3ed48-acc2-41b5-b4ac-a6107b651789, text='nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of m...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=6, score=None), Document(id=7bfd84e6-0025-4cfa-8c0a-63c9de9a8d4a, text='nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of m...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=7, score=None), Document(id=8bece98d-65f0-4dd1-9407-d1c54413bef4, text='nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of m...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=8, score=None), Document(id=cf9ab236-af73-4af6-9302-b3c7ffdd9ca7, text='nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of m...', meta_data={'title': 'Interviewing Li Yin'}, vector='len: 256', parent_doc_id=doc2, order=9, score=None)]}, transformer_setups={'data_transformer': Sequential(\n",
      "    (0): TextSplitter(split_by=word, chunk_size=400, chunk_overlap=200)\n",
      "    (1): ToEmbeddings(\n",
      "      batch_size=100\n",
      "      (embedder): Embedder(\n",
      "        model_kwargs={'model': 'text-embedding-3-small', 'dimensions': 256, 'encoding_format': 'float'}, \n",
      "        (model_client): OpenAIClient()\n",
      "      )\n",
      "      (batch_embedder): BatchEmbedder(\n",
      "        (embedder): Embedder(\n",
      "          model_kwargs={'model': 'text-embedding-3-small', 'dimensions': 256, 'encoding_format': 'float'}, \n",
      "          (model_client): OpenAIClient()\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )}, mapper_setups={}, index_path='/root/.adalflow/index.faiss')\n",
      "  (retriever): FAISSRetriever(\n",
      "    top_k=5, metric=prob, dimensions=256, total_documents=17\n",
      "    (embedder): Embedder(\n",
      "      model_kwargs={'model': 'text-embedding-3-small', 'dimensions': 256, 'encoding_format': 'float'}, \n",
      "      (model_client): OpenAIClient()\n",
      "    )\n",
      "  )\n",
      "  (retriever_output_processors): RetrieverOutputToContextStr(deduplicate=True)\n",
      "  (generator): Generator(\n",
      "    model_kwargs={'model': 'gpt-3.5-turbo', 'temperature': 0.3, 'stream': False}, trainable_prompt_kwargs=[]\n",
      "    (prompt): Prompt(\n",
      "      template: <START_OF_SYSTEM_PROMPT>\n",
      "      {# task desc #}\n",
      "      {% if task_desc_str %}\n",
      "      {{task_desc_str}}\n",
      "      {% else %}\n",
      "      You are a helpful assistant.\n",
      "      {% endif %}\n",
      "      {#input format#}\n",
      "      {% if input_format_str %}\n",
      "      <INPUT_FORMAT>\n",
      "      {{input_format_str}}\n",
      "      </INPUT_FORMAT>\n",
      "      {% endif %}\n",
      "      {# output format #}\n",
      "      {% if output_format_str %}\n",
      "      <OUTPUT_FORMAT>\n",
      "      {{output_format_str}}\n",
      "      </OUTPUT_FORMAT>\n",
      "      {% endif %}\n",
      "      {# tools #}\n",
      "      {% if tools_str %}\n",
      "      <TOOLS>\n",
      "      {{tools_str}}\n",
      "      </TOOLS>\n",
      "      {% endif %}\n",
      "      {# example #}\n",
      "      {% if examples_str %}\n",
      "      <EXAMPLES>\n",
      "      {{examples_str}}\n",
      "      </EXAMPLES>\n",
      "      {% endif %}\n",
      "      {# chat history #}\n",
      "      {% if chat_history_str %}\n",
      "      <CHAT_HISTORY>\n",
      "      {{chat_history_str}}\n",
      "      </CHAT_HISTORY>\n",
      "      {% endif %}\n",
      "      {#contex#}\n",
      "      {% if context_str %}\n",
      "      <CONTEXT>\n",
      "      {{context_str}}\n",
      "      </CONTEXT>\n",
      "      {% endif %}\n",
      "      <END_OF_SYSTEM_PROMPT>\n",
      "      <START_OF_USER_PROMPT>\n",
      "      {% if input_str %}\n",
      "      {{input_str}}\n",
      "      {% endif %}\n",
      "      <END_OF_USER_PROMPT>\n",
      "      {# steps #}\n",
      "      {% if steps_str %}\n",
      "      <START_OF_ASSISTANT_STEPS>\n",
      "      {{steps_str}}\n",
      "      <END_OF_ASSISTANT_STEPS>\n",
      "      {% endif %}\n",
      "      , prompt_kwargs: {'task_desc_str': '\\nYou are a helpful assistant.\\n\\nYour task is to answer the query that may or may not come with context information.\\nWhen context is provided, you should stick to the context and less on your prior knowledge to answer the query.\\n\\nOutput JSON format:\\n{\\n    \"answer\": \"The answer to the query\",\\n}'}, prompt_variables: ['examples_str', 'context_str', 'chat_history_str', 'tools_str', 'task_desc_str', 'input_str', 'input_format_str', 'output_format_str', 'steps_str']\n",
      "    )\n",
      "    (model_client): OpenAIClient()\n",
      "    (output_processors): JsonParser()\n",
      "  )\n",
      ")\n",
      "Response: (GeneratorOutput(id=None, data={'answer': \"Li Yin's hobby is rock climbing and profession is an AI researcher and a software engineer.\"}, error=None, usage=CompletionUsage(completion_tokens=25, prompt_tokens=2713, total_tokens=2738), raw_response='{\\n    \"answer\": \"Li Yin\\'s hobby is rock climbing and profession is an AI researcher and a software engineer.\"\\n}', metadata=None), ' My name is Li Yin, I love rock climbinglots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of  textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textLi Yin is an AI researcher and a software engineerlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more  textLi Yin is an AI researcher and a software engineerlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more  nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense  nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense ')\n"
     ]
    }
   ],
   "source": [
    "# Prepare initial documents\n",
    "doc1 = Document(\n",
    "    meta_data={\"title\": \"Li Yin's profile\"},\n",
    "    text=\"My name is Li Yin, I love rock climbing\" + \"lots of nonsense text\" * 500,\n",
    "    id=\"doc1\",\n",
    ")\n",
    "doc2 = Document(\n",
    "    meta_data={\"title\": \"Interviewing Li Yin\"},\n",
    "    text=\"lots of more nonsense text\" * 250\n",
    "    + \"Li Yin is an AI researcher and a software engineer\"\n",
    "    + \"lots of more nonsense text\" * 250,\n",
    "    id=\"doc2\",\n",
    ")\n",
    "\n",
    "# Prepare the database (only runs once)\n",
    "prepare_database_with_index([doc1, doc2], index_file=\"index.faiss\")\n",
    "\n",
    "# Initialize RAG\n",
    "rag = RAG(index_file=\"index.faiss\")\n",
    "print(rag)\n",
    "\n",
    "# Query the RAG system\n",
    "query = \"What is Li Yin's hobby and profession?\"\n",
    "response = rag.call(query)\n",
    "print(f\"Response: {response}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bcC1-dCheVEC",
    "outputId": "133bab3f-ff2e-40db-99dc-71d64af6283f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Splitting Documents in Batches: 100%|██████████| 1/1 [00:00<00:00, 114.76it/s]\n",
      "Batch embedding documents: 100%|██████████| 1/1 [00:00<00:00,  1.35it/s]\n",
      "Adding embeddings to documents from batch: 1it [00:00, 1915.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved the state of the DB to /root/.adalflow/index.faiss\n",
      "Response: (GeneratorOutput(id=None, data={'answer': \"Apple's favorite toy is her monkey toy.\"}, error=None, usage=CompletionUsage(completion_tokens=16, prompt_tokens=2647, total_tokens=2663), raw_response='{\\n    \"answer\": \"Apple\\'s favorite toy is her monkey toy.\"\\n}', metadata=None), ' Apple is a cute dog with black and tan furlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots of nonsense textlots  textApple is energetic, loves to play with her monkey toylots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more  textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textApple is energetic, loves to play with her monkey toylots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more  textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textLi Yin is an AI researcher and a software engineerlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more  textLi Yin is an AI researcher and a software engineerlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more ')\n"
     ]
    }
   ],
   "source": [
    "# Add more documents at runtime\n",
    "doc3 = Document(\n",
    "    meta_data={\"title\": \"Apple's profile\"},\n",
    "    text=\"Apple is a cute dog with black and tan fur\" + \"lots of nonsense text\" * 500,\n",
    "    id=\"doc3\",\n",
    ")\n",
    "doc4 = Document(\n",
    "    meta_data={\"title\": \"Apple's characteristics\"},\n",
    "    text=\"lots of more nonsense text\" * 250\n",
    "    + \"Apple is energetic, loves to play with her monkey toy\"\n",
    "    + \"lots of more nonsense text\" * 250,\n",
    "    id=\"doc4\",\n",
    ")\n",
    "\n",
    "rag.add_documents([doc3, doc4])\n",
    "rag.prepare_retriever()\n",
    "\n",
    "# Test a new query\n",
    "query = \"What is Apple's favorite toy?\"\n",
    "response = rag.call(query)\n",
    "print(f\"Response: {response}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "o9TzVv5GeZZ2",
    "outputId": "bde56355-186c-4013-d702-b4530f82881b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All documents in the database:\n",
      "ID: doc1, Title: Li Yin's profile, Text: My name is Li Yin, I love rock climbinglots of nonsense textlots of nonsense textlots of nonsense te...\n",
      "ID: doc2, Title: Interviewing Li Yin, Text: lots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense ...\n",
      "ID: doc3, Title: Apple's profile, Text: Apple is a cute dog with black and tan furlots of nonsense textlots of nonsense textlots of nonsense...\n",
      "ID: doc4, Title: Apple's characteristics, Text: lots of more nonsense textlots of more nonsense textlots of more nonsense textlots of more nonsense ...\n"
     ]
    }
   ],
   "source": [
    "# View all documents in the database\n",
    "print(\"All documents in the database:\")\n",
    "for item in rag.db.items:\n",
    "    print(\n",
    "        f\"ID: {item.id}, Title: {item.meta_data['title']}, Text: {item.text[:100]}...\"\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
